package com.gsy.wordcount;

import com.gsy.gsy_common_util.crawler.WebCrawlerUtil;
import com.gsy.gsy_common_util.fileUtil.DirectoryUtils;
import com.gsy.gsy_common_util.fileUtil.FileUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.*;
import java.nio.charset.Charset;
import java.util.*;

public class Clean {

    public static void main(String[] args) {
        String basedir = "D:\\test\\工科英语\\工科英语\\401";
        List<File> files = DirectoryUtils.listAllFiles(new File(basedir),"txt");
        HashSet<String> hashSet = new HashSet<>();
        try {
            String s = FileUtils.fileToString("D:\\test\\工科英语\\stop.txt");
            String []stop = s.split("\\s");
            for (String s1 : stop) {
                hashSet.add(s1.toLowerCase());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        for (File file : files) {
            try {
                String text = FileUtils.fileToString(file);
//            System.out.println(text);
                String a = text.replaceAll("[^a-zA-Z]", " ").replaceAll("\\s+"," ");
                String arr[] = a.split("\\s");
                HashMap<String, Integer> hashMap = new HashMap<>();
                for (String s : arr) {
                    if (s.matches("[A-Z][a-z]+")){
                        s = s.toLowerCase();
                    }
                    if (s.length() < 4){
                        continue;
                    }
                    if (!hashSet.contains(s.toLowerCase())){
                        hashMap.put(s, hashMap.getOrDefault(s, 0) + 1);
                    }
                }
                ArrayList<Node> arrayList = new ArrayList<>();
                for (Map.Entry<String, Integer> stringIntegerEntry : hashMap.entrySet()) {
                    arrayList.add(new Node(stringIntegerEntry.getKey(), stringIntegerEntry.getValue()));
                }
                Collections.sort(arrayList);
                StringBuffer stringBuffer = new StringBuffer();
                for (Node node : arrayList) {
//                System.out.println(node);
                    stringBuffer.append(node.toString()+"\n");
                }
                FileUtils.stringToFile(stringBuffer.toString(),file.getParent(),file.getName()+"ana.csv","utf-8");
            } catch (IOException e) {
                e.printStackTrace();
            }

        }
//        String fileName = "D:\\test\\aaa.txt";
//        File file = new File(fileName);

    }

}

class Node implements Comparable<Node> {
    String text;
    Integer value;

    public Node(String text, Integer value) {
        this.text = text;
        this.value = value;
    }

    @Override
    public int compareTo(Node o) {
        return -this.value.compareTo(o.value);
    }

    @Override
    public String toString() {
        return text + "," + value;
    }
}


